package com.samp.solr.analysis;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 根据标点符号进行分词。
 *
 */
public class PunctuationTokenizer extends CharTokenizer{
	
	private static final Logger logger = LoggerFactory.getLogger(PunctuationTokenizer.class);
	
	private String punctuations = ",.?;:!，。？；：！";
	
	  public PunctuationTokenizer() {
	  }

	  public PunctuationTokenizer(AttributeFactory factory, String newPunctuations) {
	    super(factory);
	    if( StringUtils.isNotBlank(newPunctuations) ){
	    	punctuations += newPunctuations;
	    }
	    logger.info("punctuations is "+punctuations);
	  }

	@Override
	protected boolean isTokenChar(int c) {
		return !(punctuations.indexOf(c) >= 0);
	}
	
}
